
#include "import.h"

#include <stdbool.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdarg.h>
#include <assert.h>

#include "ltree.h"

#include "common/alloc.h"
#include "common/utf8.h"

#include "str_hash_table.h"


// intermediate file buffer size
#define MAX_FILE_BUF_SIZE   (256*256)

// intermediate buffer for utf8 -to- ucs4 conversion
#define MAX_UCS4_BUF_SIZE   (32*256)



/**
 * import context used during all the import process
 */
struct ImportContext
{
    FILE * fdest;
    FILE * fallCountries;
    FILE * falternateNames;
    FILE * fcountryInfo;

    ImportFeedback verbose;
    ImportFeedback error;

    // adminitstrative units hash table
    struct HashTable * adm_units_hash_table;

    // lexemes tree
    struct Node * ltree;

    // intermediate file buffer for fread of MAX_FILE_BUF_SIZE size
    uint8_t * file_buf;

    // intermediate buffer for utf8 to ucs4 conversions of MAX_UCS4_BUF_SIZE size
    uint32_t * ucs4_buf;

    uint32_t imported_city_count;
};

static void import_verbose(struct ImportContext * ctx, char * format, ...)
{
    if (ctx->verbose)
    {
        char buf[256];

        va_list ap;
        va_start(ap, format);
        vsnprintf(buf, sizeof(buf), format, ap);
        va_end(ap);

        ctx->verbose(buf);
    }
}

static void import_error(struct ImportContext * ctx, char * format, ...)
{
    if (ctx->error)
    {
        char buf[256];

        va_list ap;
        va_start(ap, format);
        vsnprintf(buf, sizeof(buf), format, ap);
        va_end(ap);

        ctx->error(buf);
    }
}


static void add_adm_unit_entries(struct HashTable * ht)
{
    // adm unit entries are as follows:
    hash_table_add_len(ht, "ADM1");
    hash_table_add_len(ht, "ADM2");
    hash_table_add_len(ht, "ADM3");
    hash_table_add_len(ht, "ADM4");
    hash_table_add_len(ht, "ADMD");
    hash_table_add_len(ht, "LTER");
    hash_table_add_len(ht, "PCL");
    hash_table_add_len(ht, "PCLD");
    hash_table_add_len(ht, "PCLF");
    hash_table_add_len(ht, "PCLI");
    hash_table_add_len(ht, "PCLIX");
    hash_table_add_len(ht, "PCLS");
    hash_table_add_len(ht, "PRSH");
    hash_table_add_len(ht, "TERR");
    hash_table_add_len(ht, "ZN");
    hash_table_add_len(ht, "ZNB");
}


struct ImportContext * create_import_context(FILE * fdest,
                                             FILE * fallCountries,
                                             FILE * falternateNames,
                                             FILE * fcountryInfo,
                                             ImportFeedback verbose,
                                             ImportFeedback error)
{
    struct ImportContext * ctx = xmalloc(sizeof(struct ImportContext));

    ctx->fdest = fdest;
    ctx->fallCountries = fallCountries;
    ctx->falternateNames = falternateNames;
    ctx->fcountryInfo = fcountryInfo;
    ctx->verbose = verbose;
    ctx->error = error;

    ctx->adm_units_hash_table = create_hash_table();
    add_adm_unit_entries(ctx->adm_units_hash_table);

    ctx->ltree = new_node(NULL);

    ctx->file_buf = xmalloc(MAX_FILE_BUF_SIZE);
    ctx->ucs4_buf = xmalloc(MAX_UCS4_BUF_SIZE * sizeof(uint32_t));

    ctx->imported_city_count = 0;

    return ctx;
}


void free_import_context(struct ImportContext * ctx)
{
    free_node(ctx->ltree);
    xfree(ctx->file_buf);
    xfree(ctx->ucs4_buf);
    free_hash_table(ctx->adm_units_hash_table);
    xfree(ctx);
}


/**
 * read file line by line
 */
static inline void read_dbdump_lines(struct ImportContext * ctx, FILE * fsrc, int (* handle_line)(struct ImportContext * ctx, uint8_t * line))
{
    if (handle_line == NULL)
    {
        import_error(ctx, "internal error: handle_line is NULL");
        return;
    }

    uint8_t * buf = ctx->file_buf;
    uint8_t * begin = buf;
    uint8_t * end = buf + MAX_FILE_BUF_SIZE;

    for (;;)
    {
        size_t bytes_read = fread(begin, 1, end - begin, fsrc);
        if (bytes_read == 0)
        {
            break;
        }

        end = begin + bytes_read;

        uint8_t * cur_line = begin;

        for (; begin != end; ++begin)
        {
            if (*begin == '\n')
            {
                *begin = 0;

                if (handle_line(ctx, cur_line) != 0)
                {
                    return;
                }

                cur_line = begin + 1;
            }
        }

        if (cur_line < end)
        {
            // we're at the end of buffer, but we still have an unfinished line - copy it to the beginning
            size_t rest = end - cur_line;
            memmove(buf, cur_line, rest);
            begin = buf + rest;
        }
        else
        {
            begin = buf;
        }
    }
}


#define READ_NEXT_ENTRY(line, begin, end) \
begin = line; \
for (;;) \
{ \
    uint8_t ch = *line; \
    if (ch == 0) \
    { \
        end = line; \
        break; \
    } \
    if (ch == '\t') \
    { \
        end = line++; \
        break; \
    } \
    else \
    { \
        ++line; \
    } \
}


static void add_each_separated_lexeme(struct ImportContext * ctx, uint32_t * ucsbuf, uint32_t * ucsbufend)
{
    // add each space-separated lexeme
    bool subfound = false;
    uint32_t * subbegin = ucsbuf;
    for (; ucsbuf < ucsbufend; ++ucsbuf)
    {
        uint32_t c = *ucsbuf;
        if (c == ' ' || c == ',')
        {
            add_lexeme(subbegin, ucsbuf, ctx->ltree);
            subbegin = ucsbuf + 1;
            subfound = true;
        }
    }

    if (subfound)
    {
        // add last sub lexeme
        add_lexeme(subbegin, ucsbufend, ctx->ltree);
    }
}


static void import_city(struct ImportContext * ctx, uint8_t * lexeme, uint8_t * lexeme_end, bool import_whole, int32_t city_id, int32_t population)
{
    if (lexeme == lexeme_end)
    {
        return;
    }

    size_t lexeme_size = lexeme_end - lexeme;

    uint32_t * ucsbuf = ctx->ucs4_buf;

    // sanity check
    if (lexeme_size >= MAX_UCS4_BUF_SIZE)
    {
        import_error(ctx, "internal error: ucs4 buffer is too small");
        return;
    }

    // convert to ucs4
    uint32_t * ucsbufend = utf8_to_ucs4_nocheck(lexeme, lexeme_end, ucsbuf);
    if (!ucsbufend)
    {
        import_error(ctx, "internal error: invalid utf-8 sequence at %d", city_id);
        return;
    }

    // lowercase all
    ucs4_tolower(ucsbuf, ucsbufend);

    if (import_whole)
    {
        // add an entire lexeme
        add_lexeme(ucsbuf, ucsbufend, ctx->ltree);
    }

    add_each_separated_lexeme(ctx, ucsbuf, ucsbufend);

    if (city_id != 0)
    {
        // add to the city entries table
        assert(city_id > 0 && population >= 0);
    }
}


//static int max_recs = 160000;

static int handle_city_line(struct ImportContext * ctx, uint8_t * line)
{
    uint8_t * begin;
    uint8_t * end;

    do
    {
        uint8_t * city_id_str;
        uint8_t * city_name;
        uint8_t * city_name_end;
        uint8_t * latin1_city_name;
        uint8_t * latin1_city_name_end;
        uint8_t * alt_names;
        uint8_t * alt_names_end;
        uint8_t * population_str;

        READ_NEXT_ENTRY(line, begin, end); // city id
        city_id_str = begin;
        *end = 0;

        READ_NEXT_ENTRY(line, begin, end); // city name
        city_name = begin;
        city_name_end = end;

        READ_NEXT_ENTRY(line, begin, end); // latin1 city name
        latin1_city_name = begin;
        latin1_city_name_end = end;

        READ_NEXT_ENTRY(line, begin, end); // alternate names
        alt_names = begin;
        alt_names_end = end;

        READ_NEXT_ENTRY(line, begin, end); // latitude
        READ_NEXT_ENTRY(line, begin, end); // longtitude
        READ_NEXT_ENTRY(line, begin, end); // feature class
        READ_NEXT_ENTRY(line, begin, end); // feature code
        if (!hash_table_contains(ctx->adm_units_hash_table, (char *)begin, end - begin))
        {
            // this is not an entry of interest
            break;
        }

        READ_NEXT_ENTRY(line, begin, end); // country code
        READ_NEXT_ENTRY(line, begin, end); // cc2
        READ_NEXT_ENTRY(line, begin, end); // admin1
        READ_NEXT_ENTRY(line, begin, end); // admin2
        READ_NEXT_ENTRY(line, begin, end); // admin3
        READ_NEXT_ENTRY(line, begin, end); // admin4
        READ_NEXT_ENTRY(line, begin, end); // population
        population_str = begin;
        *end = 0;

        // extract, validate and record data
        int32_t city_id = atoi((char *)city_id_str);
        int32_t population = atoi((char *)population_str);

        if (population < 0)
        {
            import_error(ctx, "error: entry #%d contains invalid population: %d\n", population);
            break;
        }

        import_city(ctx, city_name, city_name_end, true, city_id, population);
        import_city(ctx, latin1_city_name, latin1_city_name_end, true, 0, 0);
        import_city(ctx, alt_names, alt_names_end, false, 0, 0);


        ++ctx->imported_city_count;

    } while (0);

    return 0;
    //return (--max_recs == 0);
}


static int handle_countryinfo_line(struct ImportContext * ctx, uint8_t * line)
{
    assert(ctx);
    fprintf(stdout, "%s", (char *)line);
    return 0;
}



void start_import(struct ImportContext * ctx)
{
    import_verbose(ctx, "import started");

    read_dbdump_lines(ctx, ctx->fallCountries, handle_city_line);

    if (!ctx->error)
    {
        read_dbdump_lines(ctx, ctx->fcountryInfo, handle_countryinfo_line);
    }

    save_tree_to_file(ctx->ltree, ctx->fdest);

    import_verbose(ctx, "import finished, %d records processed", ctx->imported_city_count);
}
